load('stocks.mat')
%load('tokens.mat')
fid = fopen('dictionary.txt','rt');
numTokens = 0;
while (fgets(fid) ~= -1),
    numTokens = numTokens+1;
end
numTokens=25;
%numTokens=length(tokens);
down_counter=0;
up_counter=0;
token_down=zeros(1,numTokens);
token_up=zeros(1,numTokens);
% fileoffset=10;
% filestr='translateddata/outputfile2009-06-';
% datestr='2009-06-10';
% findindex=0;
% for i=1:length(date)
%     if isequal(char(date(i)),datestr)
%         findindex=i
%     end
% end


path='nimaoutputs';
foldercontent=dir(path);
numDays=size(foldercontent,1)-3;
datestr_start=18;
datestr_length=9;



stocks_status=[];
for i=1:numDays
    filename=char(foldercontent(i+3).name);
    datestr=filename(datestr_start:datestr_start+datestr_length);
    findindex=0;
    for j=1:length(date)
        if isequal(char(date(j)),datestr)
            findindex=j;
        end
    end
    stocks_status=[stocks_status,y(findindex)];
    findindex
    fullfilename=[path '/' char(foldercontent(i+3).name)]

    fid = fopen(fullfilename,'rt');
    numTweets = 0;
    while (fgets(fid) ~= -1),
        numTweets = numTweets+1;
    end
    fclose(fid);
    trainMatrix = textread(fullfilename,'%d', 50*numTweets);
    trainMatrix=trainMatrix(trainMatrix>0);
    trainMatrix=trainMatrix(trainMatrix<=numTokens);
    if stocks_status(i)==0 %stock will be down
        for j=1:length(trainMatrix)
            down_counter=down_counter+numTweets;
            token_down(trainMatrix(j))=token_down(trainMatrix(j))+1;
        end
    else
        for j=1:length(trainMatrix)
            up_counter=up_counter+numTweets;
            token_up(trainMatrix(j))=token_up(trainMatrix(j))+1;
        end
    end
end

log_p_up_prior=log(up_counter/(up_counter+down_counter));     % log of prior on spam
log_p_down_prior=log(down_counter/(up_counter+down_counter)); % log of prior on non-spam
log_p_token_down=log((token_down+1)/(sum(token_down)+numTokens)); % log of prob of tokens given spam 
log_p_token_up=log((token_up+1)/(sum(token_up)+numTokens)); % log of prob of tokens given spam 
save('naivebayesresult_200.mat','log_p_up_prior','log_p_down_prior','log_p_token_down','log_p_token_up','numTokens')
[y,I]=sort(log_p_token_up,'descend');
I(1:10)

% [y,I]=sort(log_p_token_down,'descend');
% I(1:10)
